dat_o2 <- read.csv("datall.csv", header=T)########
#dat_co2 = dat_co2[(1 + 10*(i-1)):(10*i), ]
#rownmames(dat_co2)= NULL
#length(unique(dat_co2$StudyID))
#dat_co2$y <- dat_co2$ETCO2
#dat_co2$X <- 1:nrow(dat_co2)
source('Functions.R')
load('errlist_o2_k10_n_2.Rdata')
errlist102 <- errlist
load('errlist_o2_k10_n_3.Rdata')
errlist103 <- errlist
load('errlist_o2_k15_n_2.Rdata')
errlist152 <- errlist
load('errlist_o2_k15_n_3.Rdata')
errlist153 <- errlist
load('errlist_o2_k20_n_2.Rdata')
errlist202 <- errlist
load('errlist_o2_k20_n_3.Rdata')
errlist203 <- errlist
load('errlist_o2_k25_n_2.Rdata')
errlist252 <- errlist
load('errlist_o2_k25_n_3.Rdata')
errlist253 <- errlist
load('errlist_o2_k30_n_2.Rdata')
errlist302 <- errlist
load('errlist_o2_k30_n_3.Rdata')
errlist303 <- errlist
#for (i in 1:length(unique(dat_co2$StudyID))) {
# try(plot(errlist[[i]]$lno_grid, errlist[[i]]$err, type = 's', xlab = 'window length', ylab = 'reduced error'))
#}
l102 <- NULL
for (i in 1:length(unique(dat_o2$StudyID))) {
a <- errlist102[[i]]$lno_grid[which.max(errlist102[[i]]$err)]
b <- tryCatch({max(errlist102[[i]]$err)}, error = function(x)NA )
a <- ifelse(length(a) == 0, yes = NA, no = a)
l102 <- rbind(l102, data.frame(lno = a,error = b))
}
l102$k = 10
l102$n = 2
l103 <- NULL
for (i in 1:length(unique(dat_o2$StudyID))) {
a <- errlist103[[i]]$lno_grid[which.max(errlist103[[i]]$err)]
b <- tryCatch({max(errlist103[[i]]$err)}, error = function(x)NA )
a <- ifelse(length(a) == 0, yes = NA, no = a)
l103 <- rbind(l103, data.frame(lno = a,error = b))
}
l103$k = 10
l103$n = 3
l152 <- NULL
for (i in 1:length(unique(dat_o2$StudyID))) {
a <- errlist152[[i]]$lno_grid[which.max(errlist152[[i]]$err)]
b <- tryCatch({max(errlist152[[i]]$err)}, error = function(x)NA )
a <- ifelse(length(a) == 0, yes = NA, no = a)
l152 <- rbind(l152, data.frame(lno = a,error = b))
}
l152$k = 15
l152$n = 2
l153 <- NULL
for (i in 1:length(unique(dat_o2$StudyID))) {
a <- errlist153[[i]]$lno_grid[which.max(errlist153[[i]]$err)]
b <- tryCatch({max(errlist153[[i]]$err)}, error = function(x)NA )
a <- ifelse(length(a) == 0, yes = NA, no = a)
l153 <- rbind(l153, data.frame(lno = a,error = b))
}
l153$k = 15
l153$n = 3
l202 <- NULL
for (i in 1:length(unique(dat_o2$StudyID))) {
a <- errlist202[[i]]$lno_grid[which.max(errlist202[[i]]$err)]
b <- tryCatch({max(errlist202[[i]]$err)}, error = function(x)NA )
a <- ifelse(length(a) == 0, yes = NA, no = a)
l202 <- rbind(l202, data.frame(lno = a,error = b))
}
l202$k <- 20
l202$n = 2
l203 <- NULL
for (i in 1:length(unique(dat_o2$StudyID))) {
a <- errlist203[[i]]$lno_grid[which.max(errlist203[[i]]$err)]
b <- tryCatch({max(errlist203[[i]]$err)}, error = function(x)NA )
a <- ifelse(length(a) == 0, yes = NA, no = a)
l203 <- rbind(l203, data.frame(lno = a,error = b))
}
l203$k <- 20
l203$n = 3
l252 <- NULL
for (i in 1:length(unique(dat_o2$StudyID))) {
a <- errlist252[[i]]$lno_grid[which.max(errlist252[[i]]$err)]
b <- tryCatch({max(errlist252[[i]]$err)}, error = function(x)NA )
a <- ifelse(length(a) == 0, yes = NA, no = a)
l252 <- rbind(l252, data.frame(lno = a,error = b))
}
l252$k = 15
l252$n = 2
l253 <- NULL
for (i in 1:length(unique(dat_o2$StudyID))) {
a <- errlist253[[i]]$lno_grid[which.max(errlist253[[i]]$err)]
b <- tryCatch({max(errlist253[[i]]$err)}, error = function(x)NA )
a <- ifelse(length(a) == 0, yes = NA, no = a)
l253 <- rbind(l253, data.frame(lno = a,error = b))
}
l253$k = 15
l253$n = 3
l302 <- NULL
for (i in 1:length(unique(dat_o2$StudyID))) {
a <- errlist302[[i]]$lno_grid[which.max(errlist302[[i]]$err)]
b <- tryCatch({max(errlist302[[i]]$err)}, error = function(x)NA )
a <- ifelse(length(a) == 0, yes = NA, no = a)
l302 <- rbind(l302, data.frame(lno = a,error = b))
}
l302$k <- 30
l302$n = 2
l303 <- NULL
for (i in 1:length(unique(dat_o2$StudyID))) {
a <- errlist303[[i]]$lno_grid[which.max(errlist303[[i]]$err)]
b <- tryCatch({max(errlist303[[i]]$err)}, error = function(x)NA )
a <- ifelse(length(a) == 0, yes = NA, no = a)
l303 <- rbind(l303, data.frame(lno = a,error = b))
}
l303$k <- 30
l303$n = 3
df_err <- cbind( l102$error, l103$error, l152$error, l153$error, l202$error, l203$error, l252$error, l253$error, l302$error, l303$error)
ind_list <- do.call(c, apply(df_err, 1, which.max))
l_list <- list( l102, l103, l152, l153, l202, l203, l252, l253, l302, l303 )
i = 1
k <- unique((l_list[[ind_list[i]]])$k)
n <- unique((l_list[[ind_list[i]]])$n)
lno <- (l_list[[ind_list[i]]])$lno[i]
plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'na', trim = 0.66)
plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'weighted')
plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'fitted')
plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'predicted')
plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'na.weighted')
plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'na.fitted')
plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'na.predicted')
loop <- (1:length(unique(dat_o2$StudyID)))
#out_indx <- c (24, 25, 30, 36, 80, 81, 85, 105, 110, 125, 138, 147, 154, 156, 178, 181, 186, 187, 188, 193, 201, 207, 208, 210, 213, 234, 239, 262, 263)
#out_indx <- c(80, 145,159,187, 253, 262, 263)
out_indx <- c (25, 36, 42, 55, 57, 80, 88, 105, 110, 115, 187, 188, 193, 198, 203, 207, 208, 210, 234, 236, 253, 262)
loop <- loop[-out_indx]
for (i in loop) {
print( paste('Patient No.', i) )
k <- unique((l_list[[ind_list[i]]])$k)
n <- unique((l_list[[ind_list[i]]])$n)
lno <- (l_list[[ind_list[i]]])$lno[i]
try( plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'na', trim = 0.66) )
lno <- ceiling(dim(dat_o2[dat_o2$StudyID == unique(dat_o2$StudyID)[i],])[1] / lno)
legend('bottomright', paste0('window length=', lno ) )
}
## [1] "Patient No. 1"
## [1] "Patient No. 2"
## [1] "Patient No. 3"
## [1] "Patient No. 4"
## [1] "Patient No. 5"
## [1] "Patient No. 6"
## [1] "Patient No. 7"
## [1] "Patient No. 8"
## [1] "Patient No. 9"
## [1] "Patient No. 10"
## [1] "Patient No. 11"
## [1] "Patient No. 12"
## [1] "Patient No. 13"
## [1] "Patient No. 14"
## [1] "Patient No. 15"
## [1] "Patient No. 16"
## [1] "Patient No. 17"
## Error in plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, :
## no loop for break/next, jumping to top level
## [1] "Patient No. 18"
## [1] "Patient No. 19"
## [1] "Patient No. 20"
## [1] "Patient No. 21"
## [1] "Patient No. 22"
## [1] "Patient No. 23"
## [1] "Patient No. 24"
## [1] "Patient No. 26"
## [1] "Patient No. 27"
## [1] "Patient No. 28"
## [1] "Patient No. 29"
## [1] "Patient No. 30"
## [1] "Patient No. 31"
## [1] "Patient No. 32"
## [1] "Patient No. 33"
## [1] "Patient No. 34"
## [1] "Patient No. 35"
## [1] "Patient No. 37"
## [1] "Patient No. 38"
## [1] "Patient No. 39"
## [1] "Patient No. 40"
## [1] "Patient No. 41"
## [1] "Patient No. 43"
## [1] "Patient No. 44"
## [1] "Patient No. 45"
## [1] "Patient No. 46"
## [1] "Patient No. 47"
## [1] "Patient No. 48"
## [1] "Patient No. 49"
## [1] "Patient No. 50"
## [1] "Patient No. 51"
## [1] "Patient No. 52"
## [1] "Patient No. 53"
## [1] "Patient No. 54"
## [1] "Patient No. 56"
## [1] "Patient No. 58"
## [1] "Patient No. 59"
## [1] "Patient No. 60"
## [1] "Patient No. 61"
## [1] "Patient No. 62"
## [1] "Patient No. 63"
## [1] "Patient No. 64"
## [1] "Patient No. 65"
## [1] "Patient No. 66"
## [1] "Patient No. 67"
## [1] "Patient No. 68"
## [1] "Patient No. 69"
## [1] "Patient No. 70"
## [1] "Patient No. 71"
## [1] "Patient No. 72"
## [1] "Patient No. 73"
## [1] "Patient No. 74"
## [1] "Patient No. 75"
## [1] "Patient No. 76"
## [1] "Patient No. 77"
## Error in if (lno == dim(dati)[1]) { :
## missing value where TRUE/FALSE needed
## [1] "Patient No. 78"
## [1] "Patient No. 79"
## [1] "Patient No. 81"
## Error in plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, :
## no loop for break/next, jumping to top level
## [1] "Patient No. 82"
## [1] "Patient No. 83"
## [1] "Patient No. 84"
## [1] "Patient No. 85"
## Error in if (ind_begin < ind_end) { : argument is of length zero
## [1] "Patient No. 86"
## [1] "Patient No. 87"
## [1] "Patient No. 89"
## [1] "Patient No. 90"
## [1] "Patient No. 91"
## [1] "Patient No. 92"
## [1] "Patient No. 93"
## [1] "Patient No. 94"
## [1] "Patient No. 95"
## [1] "Patient No. 96"
## [1] "Patient No. 97"
## [1] "Patient No. 98"
## [1] "Patient No. 99"
## [1] "Patient No. 100"
## [1] "Patient No. 101"
## [1] "Patient No. 102"
## [1] "Patient No. 103"
## [1] "Patient No. 104"
## Error in plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, :
## no loop for break/next, jumping to top level
## [1] "Patient No. 106"
## [1] "Patient No. 107"
## [1] "Patient No. 108"
## [1] "Patient No. 109"
## [1] "Patient No. 111"
## [1] "Patient No. 112"
## [1] "Patient No. 113"
## [1] "Patient No. 114"
## [1] "Patient No. 116"
## [1] "Patient No. 117"
## [1] "Patient No. 118"
## [1] "Patient No. 119"
## [1] "Patient No. 120"
## [1] "Patient No. 121"
## [1] "Patient No. 122"
## [1] "Patient No. 123"
## [1] "Patient No. 124"
## [1] "Patient No. 125"
## [1] "Patient No. 126"
## [1] "Patient No. 127"
## [1] "Patient No. 128"
## [1] "Patient No. 129"
## [1] "Patient No. 130"
## [1] "Patient No. 131"
## [1] "Patient No. 132"
## [1] "Patient No. 133"
## [1] "Patient No. 134"
## [1] "Patient No. 135"
## [1] "Patient No. 136"
## [1] "Patient No. 137"
## [1] "Patient No. 138"
## Error in plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, :
## no loop for break/next, jumping to top level
## [1] "Patient No. 139"
## [1] "Patient No. 140"
## [1] "Patient No. 141"
## [1] "Patient No. 142"
## [1] "Patient No. 143"
## [1] "Patient No. 144"
## [1] "Patient No. 145"
## [1] "Patient No. 146"
## [1] "Patient No. 147"
## [1] "Patient No. 148"
## [1] "Patient No. 149"
## [1] "Patient No. 150"
## [1] "Patient No. 151"
## [1] "Patient No. 152"
## [1] "Patient No. 153"
## [1] "Patient No. 154"
## [1] "Patient No. 155"
## [1] "Patient No. 156"
## [1] "Patient No. 157"
## [1] "Patient No. 158"
## [1] "Patient No. 159"
## [1] "Patient No. 160"
## [1] "Patient No. 161"
## [1] "Patient No. 162"
## [1] "Patient No. 163"
## [1] "Patient No. 164"
## [1] "Patient No. 165"
## [1] "Patient No. 166"
## [1] "Patient No. 167"
## [1] "Patient No. 168"
## [1] "Patient No. 169"
## [1] "Patient No. 170"
## [1] "Patient No. 171"
## [1] "Patient No. 172"
## [1] "Patient No. 173"
## [1] "Patient No. 174"
## [1] "Patient No. 175"
## [1] "Patient No. 176"
## [1] "Patient No. 177"
## [1] "Patient No. 178"
## [1] "Patient No. 179"
## [1] "Patient No. 180"
## [1] "Patient No. 181"
## Error in plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, :
## no loop for break/next, jumping to top level
## [1] "Patient No. 182"
## [1] "Patient No. 183"
## Error in if (ind_begin < ind_end) { : argument is of length zero
## [1] "Patient No. 184"
## [1] "Patient No. 185"
## [1] "Patient No. 186"
## [1] "Patient No. 189"
## [1] "Patient No. 190"
## [1] "Patient No. 191"
## [1] "Patient No. 192"
## [1] "Patient No. 194"
## [1] "Patient No. 195"
## [1] "Patient No. 196"
## [1] "Patient No. 197"
## [1] "Patient No. 199"
## [1] "Patient No. 200"
## [1] "Patient No. 201"
## [1] "Patient No. 202"
## [1] "Patient No. 204"
## [1] "Patient No. 205"
## [1] "Patient No. 206"
## [1] "Patient No. 209"
## [1] "Patient No. 211"
## [1] "Patient No. 212"
## [1] "Patient No. 213"
## [1] "Patient No. 214"
## [1] "Patient No. 215"
## [1] "Patient No. 216"
## [1] "Patient No. 217"
## [1] "Patient No. 218"
## [1] "Patient No. 219"
## [1] "Patient No. 220"
## [1] "Patient No. 221"
## [1] "Patient No. 222"
## [1] "Patient No. 223"
## Error in if (ind_begin < ind_end) { : argument is of length zero
## [1] "Patient No. 224"
## [1] "Patient No. 225"
## [1] "Patient No. 226"
## [1] "Patient No. 227"
## [1] "Patient No. 228"
## [1] "Patient No. 229"
## [1] "Patient No. 230"
## [1] "Patient No. 231"
## [1] "Patient No. 232"
## [1] "Patient No. 233"
## [1] "Patient No. 235"
## [1] "Patient No. 237"
## [1] "Patient No. 238"
## [1] "Patient No. 239"
## [1] "Patient No. 240"
## [1] "Patient No. 241"
## [1] "Patient No. 242"
## [1] "Patient No. 243"
## [1] "Patient No. 244"
## [1] "Patient No. 245"
## [1] "Patient No. 246"
## [1] "Patient No. 247"
## [1] "Patient No. 248"
## [1] "Patient No. 249"
## [1] "Patient No. 250"
## [1] "Patient No. 251"
## [1] "Patient No. 252"
## [1] "Patient No. 254"
## [1] "Patient No. 255"
## [1] "Patient No. 256"
## [1] "Patient No. 257"
## [1] "Patient No. 258"
## [1] "Patient No. 259"
## [1] "Patient No. 260"
## [1] "Patient No. 261"
## [1] "Patient No. 263"
## [1] "Patient No. 264"
## [1] "Patient No. 265"
## [1] "Patient No. 266"
## [1] "Patient No. 267"
## [1] "Patient No. 268"
## [1] "Patient No. 269"
## [1] "Patient No. 270"
## [1] "Patient No. 271"
## Error in if (lno == dim(dati)[1]) { : argument is of length zero
loop <- 1:length(unique(dat_o2$StudyID))
#loop <- loop[!is.na(l)]
#loop <- loop[1]
clnum <- detectCores()
cl <- makeCluster(clnum)
registerDoSNOW(cl)
pb <- txtProgressBar(max = length(loop), style = 3)
progress <- function(n) setTxtProgressBar(pb, n)
opts <- list(progress = progress)
t1 <- Sys.time()
dat_o2_reduction <- foreach(x=loop,.packages=c('chron', 'refund', 'mgcv', 'fda', 'tidyverse'), .options.snow = opts, .combine = rbind ) %dopar% {
k <- unique((l_list[[ind_list[x]]])$k)
n <- unique((l_list[[ind_list[x]]])$n)
lno <- (l_list[[ind_list[x]]])$lno[x]
tryCatch({
plt_outlier_removal(data = dat_o2, co2 = F, i = x, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'na', trim = 0.6)
}, error = function(x)NA)
}
close(pb)
stopCluster(cl)
t2 <- Sys.time()
t2 - t1
#dat_o2_reduction$y <- as.numeric(dat_o2_reduction$y)
#dat_o2_reduction$survival <- as.numeric(dat_o2_reduction$survival)
write.csv(dat_o2_reduction, file = 'dat_o2_na_29_Apr_2020.csv')